=begin

  The module Readers contains functions for extracting the text content 
  from different kind of files.
 
  The only externally callable function is Readers::get_text(path)
  that given a file name returns a string with the text content, or 
  the empty string if the the file type is unknown or an error occurs

=end 

require 'rubygems'
require 'hpricot'
require 'htmlentities'
require 'iconv'
require 'kconv'
require 'open-uri'

module Readers

  class ReaderError < StandardError
  end

  def Readers.get_text(path)
  
  	if ($platform.include?("linux"))														# program is running on GNU/Linux machine 
			# provare a controllare path per html2text
		    begin
		      ext = get_ext(path)
		      case ext
		      # if this flag is set to true html2text is installed
		      #when /(html|php|asp)/: ($html2textFlag == true) ? html_reader_system(path) : html_reader(path) 
		      when /(html|php|asp)/: html_reader(path)
		      when "pdf": pdf_reader(path)
		      when "doc": word_reader(path)
		      when "txt": text_reader(path)
		      when "rb":  text_reader(path)
		      when "c":   text_reader(path)
		      else ($html2textFlag == true) ? html_reader_system(path) : html_reader(path)
		      end #case
		    rescue Readers::ReaderError => msg
		      $error_logger.error(msg)
		      return ""  
		    end
		   #end # if
		elsif ($platform.include?("mswin"))												# program is running on MS Windows machine 
			if (path =~ /^http:\/\/.*$/) != nil
	  		html_reader(path)
			else
				begin
		      ext = File.extname(path).downcase
		      case ext
		      when /(html|php|asp)/: html_reader(path)
		      when "txt": text_reader(path)
		      else unknown_type_reader(path)
		      end #case
		      rescue Readers::ReaderError => msg
		      	# send message to log file
		      	$error_logger.error(msg)
		      	return ""  
		    end #rescue
		   end #if
	  end #if
  end #get_text


  # ---------- handling of a file of unknown type 
  def Readers.unknown_type_reader(path)
    # send warning to log file
    $logger.warn("Unknown file type #{path}")
    return ""
  end

  # ---------- pdf conversion 
  def Readers.pdf_reader(path)
		begin
			remote = false															# flag used for remote/local file
			puts "call pdf_reader"
			options = "-enc Latin1 -nopgbrk"
			temp = path.scan(/\w+/)
			if temp.include?("http")
				remote = true
				temp.delete_at(0) 
			end
			# local file
			if (remote == false)
				pdf_text = %x{pdftotext #{options} #{path} -}
			# remote file
			else
				status = Timeout::timeout($timeout) {								# set a timeout for the operation
					filename = "tmp/"+temp.join("_")+".pdf"							# save temp file from remote file
		    		tempfile = File.open(filename,"w")
					tempfile.write(open(path, "r").read)
		    		tempfile.close
					pdf_text = %x{pdftotext #{options} #{filename} -}				# call pdftotext and read remote pdf file
				}
			end
			#system("pdftotext #{options} #{path} #{path}.txt 2>/dev/null")
		rescue StandardError, Timeout::Error => msg
			$logger.error("Readers::pdf_reader") {msg}
			raise ReaderError, "Failed converting pdf file: #{path} #{msg}"
		end #rescue
		# end DEBUG
		begin
			if $?.exitstatus == 127
				raise ReaderError, 'pdftotext(1) missing'
			elsif $?.exitstatus != 0
				raise ReaderError, "Failed converting pdf file: #{path}"
			else
				return fixing_invalid_utf8(pdf_text)
			end # if
		rescue StandardError => msg
			$error_logger.error(msg)
			raise ReaderError, "Failed converting pdf file: #{path}"
		end # rescue
	end


  # ----------- word conversion
  def Readers.word_reader(path)
  	#puts "call word_reader"
    options = "-m 8859-1"
    word_text = %x{antiword #{options} #{path}  2>/dev/null}
    #word_text = system("antiword #{options} #{path}  2>/dev/null")
    # puts $?.exitstatus
    if $?.exitstatus == 127
      raise ReaderError, 'antiword(1) missing'
    elsif $?.exitstatus != 0
      raise ReaderError, "Failed converting word file: #{path} #{word_text}"
    else
      return word_text
    end
  end


  # ---------- text: only utf8 -> iso-8859-1 conversion
  def Readers.text_reader(path)
    begin
    	status = Timeout::timeout($timeout) {						# set a timeout for the operation
			#puts "call text_reader"
			filename = "tmp/"+(path.hash).to_s+".txt"
			tempfile = File.open(filename,"w")
		  	tempfile.write(open(path).read)
			tempfile.close
			s = IO.read(filename)
			puts "text file readed"
			s = Iconv.conv('utf-8','iso-8859-1',s) if s.isutf8
			#system("rm "+filename)
			return s
		}
    rescue StandardError, Timeout::Error => msg
    	$logger.error("Readers::text_reader") {msg}
      raise ReaderError, "Failed converting text file: #{path}"
    end
  end


  # ---------- html coversion 
  def Readers.html_reader(path)
    begin
		doc = Hpricot(open(path))
        text = html2text(doc)
		text.gsub!(/\n\s+\n/, "\n\n")  								# remove groups of empty lines
        return fixing_invalid_utf8(text)
    rescue StandardError => msg
    	$logger.error("Readers::html_reader") {msg}
      raise ReaderError, "Failed converting html file: #{path}"
    end
  end

  def Readers.html2text(a)
    if(a.respond_to? :children) && (a.children != nil)
      a.children.map { |x| html2text(x) }.join
    elsif a.text?
      s = a.to_html  # needed to preserve entities
      htmlstring2text(s)
    end
  end

  def Readers.htmlstring2text(s)
    if !s.isutf8   # if this is not utf8 convert it
      # s.gsub!("\222","'")     # (0x92) single quote char cp1250
      # the above gsub was removed since Iconv accepts \222 as an 8859-* char 
      s = Iconv.conv('utf-8','iso-8859-1',s)
    end
    s = HTMLEntities.new.decode(s)  # convert entities to utf-8
    # go back to 8859-1 ignoring charcters without a 8859-1 correspondent
    # this way we get rid of exotic symbols...
    s = Iconv.conv('iso-8859-1//IGNORE','utf-8',s)
    # this last conversion can produce the infamuos \222=0x92 character
    # which is not an iso-8859-* character. Get rid of it! 
    s.gsub!("\222","'")     # single quote cp1250
    return s
  end

	# html to text method that uses html2text external program (only on GNU/Linux machine)	
	def Readers.html_reader_system(path)
		#puts "call html system"  																				# DEBUG puts
		text = %x{html2text #{path} 2>/dev/null}
    #text = system("html2text #{path}")
    # puts $?.exitstatus
    if $?.exitstatus == 127
      raise ReaderError, 'html2text(1) missing'
    elsif $?.exitstatus != 0
      raise ReaderError, "Failed converting html file: #{path}"
    else
      return fixing_invalid_utf8(text)
    end
  end
end

# simple method to get the extension of file pointed by passed url
	def get_ext(url)
		if url.include?(".html")
			return "html"
		elsif url.include?(".htm")
			return "html"
		elsif url.include?(".xhtml")
			return "html"
		elsif url.include?(".shtml")
			return "html"
		elsif url.include?("jsp")
			return "html"
		elsif url.include?(".php")
			return "php"
		elsif url.include?(".asp")
			return "asp"
		elsif url.include?(".aspx")
			return "asp"
		elsif url.include?(".pdf")
			return "pdf"
		elsif url.include?(".doc")
			return "doc"
		elsif url.include?(".txt")
			return "txt"
		else
			return "html"
		end #if
	end #get_ext
	
	# fixing invalid UTF8
	def fixing_invalid_utf8(text)
    	ic = Iconv.new('UTF-8//IGNORE', 'UTF-8')
		valid_text = ic.iconv(text)
		return valid_text
	end
	
if ARGV[0]
  puts Readers::get_text(ARGV[0])
end

=begin
begin
status = Timeout::timeout(TIMEOUT) { 													# wait for max TIMEOUT seconds

}
rescue Timeout::Error => msg
raise ReaderError, '#{msg} on #{path}'
end #rescue
=end
